import json

import requests
import pymysql
import os

"""
本次标识一次版本更改 网页请求发生变化
"""
headers = {'Accept': '*/*',
           'Accept-Encoding': 'gzip, deflate',
           'User-Agent': 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)',
           'Host': 'ipub.exuezhe.com',
           'DNT': '1',
           'Proxy-Connection': 'Keep-Alive',
           'Pragma': 'no-cache',

           }
proxies = {"http": "192.168.30.176:8012"}
hostUrl = "http://ipub.exuezhe.com/index.html"
cur_dir_fullpath = os.path.dirname(os.path.abspath(__file__))
HtmlRoot = os.path.join(cur_dir_fullpath, 'html')

host = '192.168.30.199'
port = 3306
user = 'root'
passwd = 'vipdatacenter'
db = 'rucjournal'


def loginWebSite(hostUrl):
    r = None
    try:
        sn = requests.Session()
        r = sn.get(hostUrl, headers=headers, proxies=proxies, timeout=30)
    except Exception as e:
        print("Login Error:" + str(e))
        return None
    if not r or r.status_code != 200:
        return None
    htmlText = r.text
    if htmlText.find("sort_nav") != -1:
        return sn


def getTaskList():
    taskList = []
    conn = pymysql.connect(host=host, port=port, user=user, passwd=passwd, db=db, charset='utf8mb4')
    cur = conn.cursor()
    sql = "select rawid,jsonmsg from article where stat =0 and failcount < 3  limit 100"
    cur.execute(sql)
    result = cur.fetchall()
    if not result:
        taskList = []
    for item in result:
        task = item[0]
        jsonmsg = item[1]
        taskList.append((task, jsonmsg))
    return taskList


def executeSqlList(sqlList):
    conn = pymysql.connect(host=host, port=port, user=user, passwd=passwd, db=db, charset='utf8mb4')
    cur = conn.cursor()
    for sql in sqlList:
        cur.execute(sql)
    conn.commit()
    print("Task update success!")


def saveFile(fileName, data):
    with open(fileName, "a", encoding="utf-8") as f:
        f.write(data)
        print(fileName + " save success")


def downloadPage(sn, url):
    r = None
    try:
        r = sn.get(url)
    except Exception as e:
        print("Download Error:" + str(e))
        return False
    if r.status_code != 200 or not r:
        return False
    htmlText = r.text
    if htmlText.find("R1") != -1:

        return htmlText
    else:
        return False


def downloadXML(sn, task, jsonmsg):
    dicts = json.loads(jsonmsg)
    TitleShow = dicts["TitleShow"]
    print("Now , download Task:" + str(task))
    baseurl = "http://ipub.exuezhe.com/Qw/GetBaseArt?id=%s&kw=%s " % (str(task), TitleShow)
    texturl = "http://ipub.exuezhe.com/Qw/GetTextArt?id=%s&pn=1&ps=2&kw=%s" % (str(task), TitleShow)
    print("baseurl -> " + baseurl)
    baseTag = downloadPage(sn, baseurl)
    if not baseTag:
        return False
    print("textUrl -> " + texturl)
    textTag = downloadPage(sn, texturl)
    if not textTag:
        textTag = ""
    htmlText = str(task) + "★" + baseTag.replace("\r", " ").replace("\n", " ").replace("\0",
                                                                                       " ") + "★" + textTag.replace(
        "\r", " ").replace("\n", " ").replace("\0", " ") + "\n"
    saveFile("detail_article.big_htm", htmlText)
    return True


if __name__ == '__main__':
    while True:
        taskList = getTaskList()
        sn = loginWebSite(hostUrl)
        sqlList = []
        if not sn:
            continue
        else:
            print("Login Success!, Now download detail pages!")
        for task, jsonmsg in taskList:
            tag = downloadXML(sn, task, jsonmsg)
            if not tag:
                sql = "update article set failcount = failcount + 1 where rawid = '%s'" % str(task)
                sqlList.append(sql)
            else:
                sql = "update article set stat = 1 where rawid = '%s'" % str(task)
                sqlList.append(sql)
        executeSqlList(sqlList)
